{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 采用5折交叉验证，分别用log似然损失和正确率，对Logistic回归模型的正则超参数调优。（各50分） "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#input data\n",
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(768, 9)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sklearn的学习器大多支持稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 默认参数的Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [ 0.48797856  0.53011593  0.4562292   0.422546    0.48392885]\n",
      "cv logloss is: 0.476159709444\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "#数据集比较大，采用3折交叉验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss') #accuracy\n",
    "#loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='accuracy') #accuracy正确率\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -(loss.mean()))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） 目标函数为：J = C* sum(logloss(f(xi), yi)) + penalty \n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "1.设置参数搜索范围\n",
    "2.生成学习器实例（参数设置）\n",
    "3.生成GridSearchCV的实例（参数设置）\n",
    "4.调用GridSearchCV的fit方法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### log似然损失，5折交叉验证 进行参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',n_jobs = 4,)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.476028846231\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "print(\"最小log损失：\", -grid.best_score_)\n",
    "print(\"最佳超参数：\", grid.best_params_) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 准确率，5折交叉验证 进行参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_penalty_accuracy= LogisticRegression(solver='liblinear')\n",
    "grid_accuracy= GridSearchCV(lr_penalty_accuracy, tuned_parameters,cv=5, scoring='accuracy',n_jobs = 4,)\n",
    "grid_accuracy.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最佳准确率： 0.774739583333\n",
      "最佳超参数： {'C': 0.1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "print(\"最佳准确率：\", grid_accuracy.best_score_)\n",
    "print(\"最佳超参数：\", grid_accuracy.best_params_) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEKCAYAAAC7c+rvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucVXW9//HXZ24MDPfbcHMEDEzwMsKgqFCZYIgEXgFD\npbKoxNTO+VWY5clOGdnxnJOaJpmJ5QVTEVLMhDKE4w1wVFAJULSB4SIqV7nO5/fHWoObYe+ZPTB7\nvjPM+/l4rMde67u+a63PXo5+XN/13d+vuTsiIiIhZIUOQEREmi4lIRERCUZJSEREglESEhGRYJSE\nREQkGCUhEREJRklIRESCURISEZFglIRERCSYnNABNHQdO3b0nj17hg5DRKRRWbx48fvu3qmmekpC\nNejZsyeLFi0KHYaISKNiZu+mU0/NcSIiEoySkIiIBKMkJCIiweidkIhICnv27KGsrIydO3eGDqXB\nys/Pp0ePHuTm5h7S8UpCIiIplJWV0apVK3r27ImZhQ6nwXF3Nm3aRFlZGb169Tqkc6g5TkQkhZ07\nd9KhQwcloBTMjA4dOhzWk6KSkIhINZSAqne490dJSESkDo2763nG3fV86DAaDSWhDNEfoojUhZYt\nW+5fHzFiBG3btmXUqFFJ606ePJni4mL69etH8+bNKS4upri4mEceeSTt682cOZNf/vKXhx13utQx\nQUSkkfjud7/Ljh07uOuuu5Lu//Wvfw3A6tWrGTVqFKWlpUnr7d27l5yc5P/5P//88+sm2DTpSUhE\npJE466yzaNWq1SEdO2TIEL7zne9QUlLC7bffzqxZszj11FM5+eSTOfvss9mwYQMAd999N9deey0A\nl156Kddccw2nn346vXv3ZubMmXX2XSrpSUhEJA03/nkZb6zdUmO9N8qjOuk0x/fr1pr/+GL/w44t\nXfv27ds/FuaHH37I6NGjMTN+85vfcMstt/CLX/zioGM2bNjAwoULef311xk7dmydPykpCYmINBHj\nxo3bv/7ee+8xduxY1q1bx65du+jbt2/SY8477zzMjBNPPJE1a9bUeUxKQiIiaUj3iaXyCWjGN07L\nZDiHpKCgYP/65MmT+cEPfsDIkSOZO3cuU6dOTXpMs2bN9q+7e53HpHdCIiJN0ObNm+nevTvuzvTp\n04PFoSQkItJIDB06lIsvvph58+bRo0cPnn766UM+149//GPOP/98Bg0aRGFhYR1GWTtqjhMRacC2\nbdu2f/25555L65iePXuydOnSA8oWLFhwwPaFF17IhRdeeNCxX/va1/av//GPf0wZS11REhIRqUMN\n8V1QQ6bmOBERCUZJSEREglESEhGRYJSEREQkGCUhEZG69Ptzo0XSEiQJmVl7M3vGzFbEn+2S1DnW\nzEoTli1mdm3C/m+b2VtmtszMbo7LeprZxwnH/Cah/kAze93MVprZraaZqkSkEaicyqG0tJTTTjuN\n/v37c+KJJzJjxoyD6moqh/RNAea5+1QzmxJvfz+xgrsvB4oBzCwbWAPMjLfPBMYAJ7n7LjPrnHDo\nKncvTnLNO4GvAy8Cc4ARwFN1+q1ERDKkRYsW3HffffTp04e1a9cycOBAvvCFL9C2bdv9dTSVQ/rG\nAJXjREwHzquh/llEyeXdePtbwFR33wXg7huqO9jMugKt3f0FjwY/ui+Na4qINBh9+/alT58+AHTr\n1o3OnTuzcePGtI/XVA4HKnT38nh9HVDTmBHjgQcTtvsCQ83sZ8BO4P+5+8vxvl5mVgpsBn7o7s8B\n3YGyhOPL4jIRkfQ8NQXWvV5zvXWvRZ/pvBfqcgKck3zg0Oq89NJL7N69m2OOOaZWxzWpqRzMbC7Q\nJcmu6xM33N3NLOXQrGaWB4wGrksozgHaA4OBQcDDZtYbKAeK3H2TmQ0EHjezWk/WYWaTgEkARUVF\ntT1cRCRjysvLueyyy5g+fTpZWbVrzGpSUzm4+7BU+8xsvZl1dffyuKmsuua0c4Al7r4+oawMeCxu\nWnvJzCqAju6+EahsoltsZquInprWAD0Sju8Rl6WKfRowDaCkpKTuxy4XkcYn3SeWyiegrzxZ5yFs\n2bKFc889l5/97GcMHjy41sdrKodPzAYmxusTgVnV1L2EA5viAB4HzgQws75AHvC+mXWKOzEQPxn1\nAd6Om/62mNnguFfc5TVcU0SkQdm9ezfnn38+l19+ORdddNFhn6+pT+UwFRhuZiuAYfE2ZtbNzOZU\nVjKzAmA48FiV4+8BepvZUuAhYGL8VPQZ4LX4ndAjwDfd/YP4mCuBu4GVwCrUM05EGpGHH36Y+fPn\nc++99+7vep2q91s6GspUDpaJx6sjSUlJiVe+yKuNhjy7ooik58033+S4446r3UEZbI5rqJLdJzNb\n7O4lNR2rqRxEROpSE0o+dUHD9oiISDBKQiIi1dAri+od7v1REhIRSSE/P59NmzYpEaXg7mzatIn8\n/PxDPofeCYmIpNCjRw/KyspqNTxOU5Ofn0+PHj1qrpiCkpCISAq5ubn06tUrdBhHNDXHiYhIMEpC\nIiISjJKQiIgEoyQkIiLBKAmJiEgwSkIiIhKMkpCIiASj3wllyA2bvhuvLQgah4hIQ6YnIRERCUZJ\nSEREglESEhGRYJSEREQkGCUhEREJRklIRESCURISEZFggiQhM2tvZs+Y2Yr4s12SOseaWWnCssXM\nrk3Y/20ze8vMlpnZzXHZhCrHVJhZcbzvWTNbnrCvc/19YxERSSbUj1WnAPPcfaqZTYm3v59Ywd2X\nA5UJJBtYA8yMt88ExgAnufuuyoTi7vcD98d1TgAed/fShNNOcPdFGf1mIiKStlDNcWOA6fH6dOC8\nGuqfBaxy93fj7W8BU919F4C7b0hyzCXAQ3UQq4iIZEioJFTo7uXx+jqgsIb644EHE7b7AkPN7EUz\n+4eZDUpyzLgqxwBMj5vifmRmdkiRi4hInclYc5yZzQW6JNl1feKGu7uZeTXnyQNGA9clFOcA7YHB\nwCDgYTPr7e4eH3MqsMPdlyYcM8Hd15hZK+BR4DLgvhTXnARMAigqKqr2e4qIyKHLWBJy92Gp9pnZ\nejPr6u7lZtYVSNacVukcYIm7r08oKwMei5POS2ZWAXQENsb7qz454e5r4s+tZvYAcAopkpC7TwOm\nAZSUlKRMkCIicnhCNcfNBibG6xOBWdXUvYSDm9UeB84EMLO+QB7wfrydBYwl4X2QmeWYWcd4PRcY\nBSxFRESCCpWEpgLDzWwFMCzexsy6mdmcykpmVgAMBx6rcvw9QG8zW0qUbCZWNsUBnwH+5e5vJ9Rv\nBjxtZq8BpUQ97X5b919LRERqI0gXbXffRNTjrWr5WmBkwvZ2oEOSeruBS1Oc+1mid0WJZduBgYcV\ntIiI1DmNmCAiIsEoCYmISDBKQiIiEoySkIiIBKMkJCIiwSgJiYhIMEpCIiISjJKQiIgEoyQkIiLB\nKAmJiEgwSkIiIhKMkpCIiASjJCQiIsEoCYmISDBKQiIiEoySkIiIBKMkJCIiwSgJScPw+3OjRUSa\nlCDTe4tUtax8MwD9A8chIvVLT0IiIhKMkpCIiAQTJAmZWXsze8bMVsSf7ZLUOdbMShOWLWZ2bbxv\nRkL5ajMrTTjuOjNbaWbLzewLCeUDzez1eN+tZmaZ/I7lFW3Z6WrtFBGpTqgnoSnAPHfvA8yLtw/g\n7svdvdjdi4GBwA5gZrxvXMK+R4HHAMysHzCe6NXCCOAOM8uOT3kn8HWgT7yMyNSX27Ovght2jOPf\nt0/k7Y3bMnUZEZFGL1QSGgNMj9enA+fVUP8sYJW7v5tYGD/NjAUeTDjvQ+6+y93fAVYCp5hZV6C1\nu7/g7g7cl8Y1D1ludhbfzP8rH3hLRt++kCdfK8/UpUREGrVQSajQ3Sv/y7wOKKyh/ng+STSJhgLr\n3X1FvN0d+FfC/rK4rHu8XrU8KTObZGaLzGzRxo0bawgtuYE573BrwT30KWzJ5AeWcOOfl7F7b8Uh\nnUtE5EiVsSRkZnPNbGmSZUxivfjJxKs5Tx4wGvhTkt2XkDw5HRZ3n+buJe5e0qlTp0M+T6esrcyY\ndBpfPaMXv1+4mnHTnmfNRx/XYaQiIo1bxt6cu/uwVPvMbL2ZdXX38ripbEM1pzoHWOLu66ucIwe4\ngOh9UaU1wFEJ2z3isjXxetXyjMvLyeKGL/ajpGc7vvfIa4y69Tn+Z1wxnzu2c31cXkSkQQvVHDcb\nmBivTwRmVVM31dPOMOAtd09sZpsNjDezZmbWi6gDwktx098WMxscv0e6vIZr1rmRJ3Rl9lVnUNg6\nn6/c+zK3/HU5+ypSPgCKiDQJoZLQVGC4ma0gSiZTAcysm5nNqaxkZgXAcOLeb1Uc9J7I3ZcBDwNv\nAH8BJrv7vnj3lcDdRJ0VVgFP1eUXSkfvTi2ZeeUZXDSgB7f9bSWX3/Mi72/bVd9hyJFAwxzVyrKb\nhrDspiGhw2g06vN+Bfkhi7tvIurxVrV8LTAyYXs70CHFOb6covxnwM+SlC8Cjj+0iOtO87xsfnnx\nSQzq1Z4fPb6Uc299jtu/NIBBPduHDk1EpN7V+knIzLLMrHUmgmlKxpYcxcwrz6B5bjbjp73AtPmr\niPpoiIg0HWklITN7wMxax81jS4E3zOy7mQ3tyNevW2tmf3sIZ/cr5KY5b/GNPyxm88d7QoclIlJv\n0n0S6ufuW4h+4PkU0Au4LGNRNSGt83O5Y8IAbhjVj7+9tYEv3raApWs2hw5LGrhl5Zv3jzwu0pil\nm4RyzSyXKAnNdvc9VPPbHqkdM+OrQ3ox4xunsWdfBRfc+X888OJ7ap4TkSNeuknoLmA1UADMN7Oj\ngS2ZCqqpGnh0O568eiiDe3fgBzNf598efpUdu/eGDktEJGPSSkLufqu7d3f3kR55Fzgzw7E1Se0L\n8rj3y4P4t+F9ebx0Def9eiErN2gQVBE5MqXbMeGauGOCmdnvzGwJ8PkMx9ZkZWUZV5/Vhz989VQ2\nbdvN6NsXMPvVtaHDEhGpc+k2x3017phwNtCOqFPC1IxFJQAM6dORJ68eSr+urbn6wVe4YdZSdu3d\nV/OBIiKNRLpJqHICuJHAH+KRCTI6KZxEurTJ58FJg5n0md7c9/y7jP3N8/zrgx2hwxIRqRPpJqHF\nZvZXoiT0tJm1AjQvQT3Jzc7iByOP467LBvL2+9sZddsC5r25vuYDRUQauHST0BVEs58OcvcdQB7w\nlYxFJUl9oX8Xnvj2EHq0a84V0xdx81/eYu8+/b+AiDRe6faOqyCa/uCHZvZfwOnu/lpGI5Okju5Q\nwKPfOp1LTinijmdXMeHuF9mwdWfosEREDkm6veOmAtcQjU79BnC1md2UycAktfzcbH5+wQnccvFJ\nvFr2EefeuoAX3t4UOiwRkVpLtzluJDDc3e9x93uAEcCozIUl6bhwYA9mTR5Cq/wcvvTbF7jj2ZVU\naI4iEWlEajOKdtuE9TZ1HYgcmmO7tGL2VUMYeUJXbv7Lcr5+3yI+2rE7dFgiImlJNwn9HHjFzO41\ns+nAYpLM2SNhtGyWw22XnMxPxvRn/oqNnHvrAl7910ehwxIRqVG6HRMeBAYTzXD6KHCau8/IZGBS\nO2bG5af15E/fPB2Ai3/zPH94frUGQRWRBq3aJGRmAyoXoCtQFi/d4jJpYIqPassT3x7CGZ/qwI9m\nLePqh0rZvkuDoIpIw1TT9N63VLPP0fhxDVK7gjx+N3EQd/5jFbf8dTlvrN3MnZcOpG9hq9ChiYgc\noNok5O4aKbuRysoyJp/5KU4uasvVD5Yy5vaF3HTB8Zx/co/QoYmI7Jfu74QuSLKcZWadD+WiZtbe\nzJ4xsxXxZ7skdY41s9KEZYuZXRvvm5FQvtrMSuPy4Wa22Mxejz8/n3C+Z81secJxhxR7Y3P6MR2Z\nc/UQTuzRhu/MeJXrHnudnXs0CKqINAw1NcdVugI4Dfh7vP05oh5yvczsJ+7+h1pedwowz92nmtmU\nePv7iRXcfTlQDGBm2cAaYGa8b1xlPTO7Baic5/h94IvuvtbMjgeeBronnHaCuy+qZayNXufW+dz/\ntVO55Zl/cuezq3it7CPunDCQog4tQocmIk1cul20c4Dj3P1Cd78Q6Ef0TuhUqiSPNI0Bpsfr04mm\nDa/OWcCqeDK9/czMgLHAgwDu/oq7V068swxobmbNDiG+I05OdhbfH/FpfjexhH99sINzb3uOvy5b\nFzosEWni0k1CR7l74rDNG+KyD4A9h3DdQncvj9fXAYU11B9PnGiqGAqsd/cVSfZdCCxx910JZdPj\nprgfxQmsyTnruEKevHooPTsUMOkPi/n5nDfZo0FQRSSQdJvjnjWzJ4A/xdsXxWUFQNJfRZrZXKBL\nkl3XJ264u5tZyh+zmFkeMBq4LsnuS0iSnMysP/ALokn4Kk1w9zXxNBSPEk3Md1+Ka04CJgEUFRWl\nCq3ROqp9Cx751mn89Ik3uWv+2yx570Nu/9IAClvnhw5NRJqYdJ+EJgO/J3pHU0zUhDbZ3ben6kHn\n7sPc/fgkyyxgvZl1BYg/N1Rz7XOInmgOmEDHzHKAC4AZVcp7EL07utzdVyXEsyb+3Ao8AJyS6oLu\nPs3dS9y9pFOnTtWE1ng1y8nmP887nl+NL2bZ2i2ce+tzLFz5fuiwRKSJSXfEBAcWAH8D5gHz/fB+\nij8bmBivTwRmVVM36dMOMAx4y93LKgvMrC3wJDDF3RcmlOeYWcd4PZdo8NWlhxH/EWNMcXdmX3UG\n7VrkcdnvXuS2eSs0CKqI1Jt0u2iPBV4iaoYbC7xoZhcdxnWnAsPNbAVRMpkaX6ebmc1JuG4BMJxo\nuKCqkr0nugr4FHBDla7YzYhmhH0NKCXqaffbw4j/iPKpzq2YddUZjD6pG7c880++cu/LfLBdg6CK\nSOal+07oeqJZVTcAmFknYC7wyKFc1N03EfV4q1q+lmjaiMrt7UCHFOf4cpKynwI/TXHZgYcSa1PR\nIi+H/xlXzKBe7blx9huMuvU5bp8wgAFFB/2ES0SkzqT7TiirMgHFNtXiWGkkzIwJpx7No986nexs\nY9xdz/P7he9oEFQRyZh0E8lfzOxpM/uymX2Z6L3LnBqOadJaVmyhZcUWWFsKW9dDReMZpeCEHm14\n4qqhfLZvZ2788xtc9cArbN15KD3xRUSql1ZznLt/18wuBM6Ii6a5+8zMhdX4ddlbTjN2w7TPRgWW\nDS0LoVUXaNU19WeL9tAAfsLUpkUuv718INPmv83NTy/njfIt3HnpAD7dpXXo0ETkCJLuOyHc/VGi\n39dIGlbn9ibX99D7wp/A1nLYui5eyuHD1fDe8/DxBwcfmJ1Xc6Jq1QWatc54sjIzvvHZYzi5qB1X\nPbCE8369kP8cczwXlxyV0euKSNNRbRIys61Ew/MctIuo57b+tziFvZbLXsuF40alrrRnJ2xb/0ly\nOuBzLWx4E1b9HXZtOfjY3BbpJau8gsP+Lqf0as+TVw/lmode4buPvMbLqz/gJ2OOJz83+7DPLSJN\nW01TOWgCmkzKzYd2R0dLdXZti5NV1UQVf659BbbMgb0fH3xss9ZxUqomUbXsEsVSjU6tmvGHK07l\nf+f+k9v+tpLX12zhjgkD6NXx8JOciDRdaTfHSUDNWkZLh2NS13GPnpiSPlXFn+89H33uS/IboObt\nqiSng9ezW3bm388+lgFHt+M7M0oZfdsCfnnxiYw4vmvmvruIHNGUhI4UZpDfJlo6HZu6njt8/GGc\nmKomq3h94/Jo3av26DMo6MSZrbrwwtGdmV+ewxsPtWDvMX0YMbiYnDbdooRV0BGy1FQnIjVTEmpq\nzKIeeC3aQ2H/1PUq9sH291M+VeVvLWd49jqG52zE3nVInGSjkfUEFJFwlIQkuaxsaFUYLSkYwL49\nzHt5Kb+ds5Cu2R9x5cAC+jTfln5PwJbR+6oee96NOnI8999Rgmqd0CRYDz0BRSQMJSE5PNm5nDX4\nZHoe04cr/7iEsxdu5erP9+Hq0X3IzkpIHNX2BCwn33eR49tg3o0HX6PanoAJ63maKVaksVESkjpx\nTKeWPD75DH74+FJ+NW8Fi9/9kF+NL6ZDy3hi2xp6Aq68aQgA/f/f09V3rqi2J2Cb5D0BE5+qWhZC\njibbFWkolISkzjTPy+a/Lj6RU3q144ZZyzj31gXc/qWTKenZPv2T5BVEvQDT6Qm4JVnnivjz3f+L\n1iuSDDfUokNCkkpMWN0+2S7oBNn610Mk0/RvmdQpM2PcoCKO796GK+9fwvhpLzDlnE9zxZBe1NmM\n6ok9ATt/OnW9ioq4J+Da1E9X65dFzYReZYpzy4KCzsmbAVsnJKvm7SFLY/mKHColIcmI/t3a8Odv\nD+F7f3qNnz75Ji+v/oCbLzqJNs1z6y+IrCwo6BAtXU5IXa9iH2zbkPqpanMZlL0MO5LMPJuVm/BE\nlex9Vbyd30adK0SSUBKSjGmdn8udlw7gdwveYepTbzH69gXcMWEA/bu1CR3agbKyo/dGrWv40e3e\n3bBtXeqnqvdXwDvzYefmg4/NaZ66c0XiO6s6GGZJpDFREpKMMjO+NrQ3Jxe1ZfL9r3D+Hf/HT0b3\nZ9ygo+quea6+5ORB26Joqc7uHamfqraug/JX4Z9/gT07Dj622mGWun2yT+QIoSQk9WLg0e158uoh\nXDujlCmPvc7Lqz/kp+cdT/O8I3BkhbwWaXau2Jpi5Iqah1k6lmz2kQ23npzBL9JwOFDhjnv0WeHg\nVT6j/Ynrn9Rts3c3jlF2YzXvEGW/1hV7Adi1cwfN8jP70wclIak3HVo2496vnMJtf1vBr+atYOma\nzdxx6QCO6dQydGj1zwzyW0dLOsMsbTmwc8WWf9xBFvto1r1hzlrvwL4KZ+++CvZWOHv3OXsrKti7\nz9lTUcG+fc7eCmfPvoqoXoWzJ6HOJ5/ROfalObuvATnZWeRkGTlZWeRkGzlZhu95nywqyGulcQ7T\nseejcgAKLfOdbpSEpF5lZxnXDuvLwKPbcc1D0SCov7joRHqFDqyhShxmqcvx+4vLF84CoO2Fd9fZ\npXbvrWD7rr1s27WX7bv3xuv7PinbdWBZYt2DynbtpSLNWeFb5GVT0CyHls1yKMjPpiAvXo+Xls0S\n9ieW5eUcUN6yWQ75uVlJm3mX3TQEyKL/v2lKtHQsi3+3l9es+tH164KSkAQxtE8nnrx6CFc98ApX\nPfAKX8wbzhXN5oUOq1GpcNi6cw/bd+2rkiRSJ4bEup8km6hs996Kmi8K5GVnUVAlMbRpnkv3tvlJ\nEkN2QuI4uKwgL+fAkTWkyQmShMysPTAD6AmsBsa6+4dV6hwb16nUG7jB3f/XzGYAlW0YbYGP3L3Y\nzHoCbwLL430vuPs34/MNBO4FmgNzgGvc03zGl4zo2qY5D00azC+eeou7F8ATuwdi1z2Z8evWR4eI\nTF/BK74XvRP68V9rjsWgZV7lU0T2/mRwVEGLeD1OKFUSSGLdxLJmOUfgezwJJtST0BRgnrtPNbMp\n8fb3Eyu4+3KgGMDMsoE1wMx437jKemZ2C5DYJ3aVuxcnueadwNeBF4mS0Ajgqbr6QnJocrOz+OGo\nfhQt+Tlv7D2KTkMmZvR69fG/HZ50MuK6tXHhfWTj9Br2tU+SxAFJJEogLfNzaJ6b3fh6IkqTESoJ\njQE+F69PB56lShKq4iyi5JI4YQAW/Zs1Fvh8dRczs65Aa3d/Id6+DzgPJaEGY2DOOwzMeYf+Z98U\nOpRGYdni+QD0/8zPA0cicnhCjTdS6O7l8fo6IPV8AZHxwINJyocC6919RUJZLzMrNbN/mNnQuKw7\nUJZQpywuExGRgDL2JGRmc4Fkv6q7PnHD3d3MUrZfmFkeMBq4LsnuSzgwOZUDRe6+KX4H9LiZVTNz\nW8prTgImARQV1fDDRBEROWQZS0LuPizVPjNbb2Zd3b08birbUM2pzgGWuPv6KufIAS4A9v9Qwt13\nAbvi9cVmtgroS/Q+qUfC4T3islSxTwOmAZSUlKjzgohIhoRqjpsNVL6BngjMqqZu1aedSsOAt9x9\nfzObmXWKOzFgZr2BPsDbcdPfFjMbHL9HuryGa4qISD0IlYSmAsPNbAVRMpkKYGbdzGxOZSUzKwCG\nA48lOUey90SfAV4zs1LgEeCb7l45r/SVwN3ASmAV6pQgIhJckN5x7r6JqMdb1fK1wMiE7e1AhxTn\n+HKSskeBpD+JdvdFwPHJ9omISBiajUtERIJREhIRkWCUhEREJBglIRERCUZJSEREglESEhGRYJSE\nREQkGCUhEREJRklIRESCURISEZFgQk1qd8Tr37VN6BBERBo8PQmJiEgwSkIiIhKMkpCIiASjJCQi\nIsEoCYmISDBKQiIiEoySkIiIBKMkJCIiwSgJiYhIMEpCIiISTJAkZGbtzewZM1sRf7ZLUudYMytN\nWLaY2bXxvhkJ5avNrDQun1DlmAozK473PWtmyxP2da7fby0iIlWFGjtuCjDP3aea2ZR4+/uJFdx9\nOVCZQLKBNcDMeN+4ynpmdguwOS6/H7g/Lj8BeNzdSxNOO8HdF2XqS4mISO2Eao4bA0yP16cD59VQ\n/yxglbu/m1hoZgaMBR5McswlwEOHGaeIiGRQqCRU6O7l8fo6oLCG+uNJnmiGAuvdfUWSfeOSHDM9\nbor7UZzAREQkoIw1x5nZXKBLkl3XJ264u5uZV3OePGA0cF2S3ZeQJDmZ2anADndfmlA8wd3XmFkr\n4FHgMuC+FNecBEwCKCoqShWaiIgcpowlIXcflmqfma03s67uXm5mXYEN1ZzqHGCJu6+vco4c4AJg\nYJJjDnpycvc18edWM3sAOIUUScjdpwHTAEpKSlImSBEROTyhmuNmAxPj9YnArGrqJn3aAYYBb7l7\nWWKhmWURvSd6KKEsx8w6xuu5wCgg8SlJREQCCJWEpgLDzWwFUTKZCmBm3cxsTmUlMysAhgOPJTlH\nqvdEnwH+5e5vJ5Q1A542s9eAUqKedr+tiy8iIiKHLkgXbXffRNTjrWr5WmBkwvZ2oEOKc3w5Rfmz\nwOAqZdtJ3mwnIiIBacQEEREJRklIRESCCTVigsgBftLhlwDMCByHiNQvPQmJiEgwSkIiIhKMkpCI\niASjJCTzIMqXAAAIvElEQVQiIsGoY4JII6SOHLXT/wcLQofQqNTn/dKTkIiIBKMkJCIiwSgJiYhI\nMHonJNIIzfjGaaFDEKkTehISEZFglIRERCQYJSEREQlGSUhERIJREhIRkWCUhEREJBglIRERCUZJ\nSEREglESEhGRYIIkITNrb2bPmNmK+LNdkjrHmllpwrLFzK6N9xWb2Qtx+SIzOyXhuOvMbKWZLTez\nLySUDzSz1+N9t5qZZfRLfuXJaBERkZRCPQlNAea5ex9gXrx9AHdf7u7F7l4MDAR2ADPj3TcDN8b7\nboi3MbN+wHigPzACuMPMsuNj7gS+DvSJlxEZ+m4iIpKmUEloDDA9Xp8OnFdD/bOAVe7+brztQOt4\nvQ2wNuG8D7n7Lnd/B1gJnGJmXYHW7v6CuztwXxrXFBGRDAs1gGmhu5fH6+uAwhrqjwceTNi+Fnja\nzP6LKJGeHpd3B15IqFcWl+2J16uWJ2Vmk4BJAEVFRTWEJiIihypjScjM5gJdkuy6PnHD3d3MvJrz\n5AGjgesSir8FfMfdHzWzscDvgGGHH/X+mKYB0wBKSkpSxiZ1R6NCizRNGUtC7p4yKZjZejPr6u7l\ncVPZhmpOdQ6wxN3XJ5RNBK6J1/8E3B2vrwGOSqjXIy5bE69XLRcRkYBCvROaTZRIiD9nVVP3Eg5s\nioPoHdBn4/XPAysSzjvezJqZWS+iDggvxU1/W8xscNwr7vIarikiIvUg1DuhqcDDZnYF8C4wFsDM\nugF3u/vIeLsAGA58o8rxXwd+ZWY5wE7i9zfuvszMHgbeAPYCk919X3zMlcC9QHPgqXgREZGALOos\nJqmUlJT4okWLQochItKomNlidy+pqZ5GTBARkWCUhEREJBglIRERCUZJSEREglESEhGRYNQ7rgZm\ntpGoG/mh6Ai8X4fh1BXFVTuKq3YUV+0cqXEd7e6daqqkJJRBZrYonS6K9U1x1Y7iqh3FVTtNPS41\nx4mISDBKQiIiEoySUGZNCx1ACoqrdhRX7Siu2mnScemdkIiIBKMnIRERCUZJqA6Z2cVmtszMKsws\nZa8SMxthZsvNbKWZTamHuNqb2TNmtiL+bJei3moze93MSs0sY6O21vT9LXJrvP81MxuQqVhqGdfn\nzGxzfH9KzeyGeojpHjPbYGZLU+wPda9qiqve71V83aPM7O9m9kb87+I1SerU+z1LM64Qf1/5ZvaS\nmb0ax3VjkjqZvV/urqWOFuA44FjgWaAkRZ1sYBXQG8gDXgX6ZTium4Ep8foU4Bcp6q0GOmY4lhq/\nPzCSaKoNAwYDL9bDP7t04voc8EQ9/019BhgALE2xv97vVZpx1fu9iq/bFRgQr7cC/tlA/r7SiSvE\n35cBLeP1XOBFYHB93i89CdUhd3/T3ZfXUO0UYKW7v+3uu4GHgDEZDm0MMD1enw6cl+HrVSed7z8G\nuM8jLwBt4xl4Q8dV79x9PvBBNVVC3Kt04grC3cvdfUm8vhV4E+hepVq937M046p38T3YFm/mxkvV\njgIZvV9KQvWvO/CvhO0yMv/HWOjR7LIA64DCFPUcmGtmi81sUoZiSef7h7hH6V7z9LhJ4ikz65/h\nmNIR4l6lK+i9MrOewMlE/3efKOg9qyYuCHDPzCzbzEqBDcAz7l6v9yvUzKqNlpnNBbok2XW9uweb\nMry6uBI33N3NLFWXyCHuvsbMOgPPmNlb8f/xSmQJUOTu28xsJPA40RTycrCg98rMWgKPAte6+5b6\num5NaogryD3zaPbpYjNrC8w0s+PdPem7vkxQEqoldx92mKdYAxyVsN0jLjss1cVlZuvNrKu7l8eP\n0RtSnGNN/LnBzGYSNVHVdRJK5/tn5B4dblyJ/9Fw9zlmdoeZdXT3kON+hbhXNQp5r8wsl+g/9Pe7\n+2NJqgS5ZzXFFfrvy90/MrO/AyOAxCSU0ful5rj69zLQx8x6mVkeMB6YneFrzgYmxusTgYOe2Mys\nwMxaVa4DZ3PgH2JdSef7zwYuj3vlDAY2JzQnZkqNcZlZFzOzeP0Uon9/NmU4rpqEuFc1CnWv4mv+\nDnjT3f87RbV6v2fpxBXinplZp/gJCDNrDgwH3qpSLbP3qz57YhzpC3A+UXvpLmA98HRc3g2Yk1Bv\nJFHvmFVEzXiZjqsDMA9YAcwF2leNi6hX2KvxsiyTcSX7/sA3gW/G6wb8Ot7/Oil6GgaI66r43rwK\nvACcXg8xPQiUA3viv60rGsi9qimuer9X8XWHEL3bfA0ojZeRoe9ZmnGF+Ps6EXgljmspcEOSv/uM\n3i+NmCAiIsGoOU5ERIJREhIRkWCUhEREJBglIRERCUZJSEREglESEgnMzLbVXKva4x8xs97xeksz\nu8vMVsXDLz1rZqeaWZ6ZzTcz/UBdGhQlIZFGLB5fLNvd346L7iYaWLSPuw8EvkI0Mvpuot+KjQsT\nqUhySkIiDUT8i/RfmtlSi+Z1GheXZ8VDuLxl0XxQc8zsoviwCcQjYJjZMcCpwA/dvQLA3d9x9yfj\nuo/H9UUaDD2aizQcFwDFwElAR+BlM5sPnAH0BPoBnYmmAbgnPuYMotELAPoDpR4NSJnMUmBQRiIX\nOUR6EhJpOIYAD7r7PndfD/yDKGkMAf7k7hXuvg74e8IxXYGN6Zw8Tk67K8cIFGkIlIREGrePgfx4\nfRlwkpllV1O/GbAz41GJpElJSKTheA4YF08y1oloCu2XgIXAhfG7oUKiaaArvQl8CsDdVwGLgBsT\nRmPuaWbnxusdgPfdfU99fSGRmigJiTQcM4lGM34V+Bvwvbj57VGikarfAP5INPnZ5viYJzkwKX2N\naObclWa2FLiXT+aPOjOuL9JgaBRtkUbAzFp6NONmB6KnozPcfV08B8zf4+1UHRIqz/EYMMXd/1kP\nIYukRb3jRBqHJ+LJx/KA/4yfkHD3j83sP4DuwHupDo4n6ntcCUgaGj0JiYhIMHonJCIiwSgJiYhI\nMEpCIiISjJKQiIgEoyQkIiLBKAmJiEgw/x+ZO0/VEz0eMQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x259a96c1dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从图中可以看到，在 log(C)取0的时候（C=1） 并且蓝色的线即 l1 正则时log损失最小"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存模型，用于后续测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as cPickle\n",
    "\n",
    "cPickle.dump(grid.best_estimator_, open(\"logloss_l1.pkl\", 'wb'))\n",
    "cPickle.dump(grid_accuracy.best_estimator_, open(\"accuracy_l2.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
